package edu.cn.nlsde.tmfst.w2v;

//import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
//import org.deeplearning4j.models.embeddings.wordvectors.WordVectors;
//import org.deeplearning4j.models.word2vec.Word2Vec;
//import org.deeplearning4j.text.sentenceiterator.LineSentenceIterator;
//import org.deeplearning4j.text.sentenceiterator.SentenceIterator;
//import org.deeplearning4j.text.sentenceiterator.SentencePreProcessor;
//import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess;
//import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.EndingPreProcessor;
//import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
//import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;

import java.io.File;
import java.io.IOException;

/**
 * Created by dell on 2016/6/30.
 */
public class W2V {
//    public static WordVectors trian_model(String data) throws IOException {
//        String path = data + ".model";
//
//        if (!(new File(path)).exists()) {
//            SentenceIterator iter = new LineSentenceIterator(new File(data));
//            iter.setPreProcessor(new SentencePreProcessor() {
//                @Override
//                public String preProcess(String sentence) {
//                    return sentence.toLowerCase();
//                }
//            });
//
//            final EndingPreProcessor preProcessor = new EndingPreProcessor();
//            TokenizerFactory tokenizer = new DefaultTokenizerFactory();
//            tokenizer.setTokenPreProcessor(new TokenPreProcess() {
//                @Override
//                public String preProcess(String token) {
//                    token = token.toLowerCase();
//                    String base = preProcessor.preProcess(token);
//                    return base;
//                }
//            });
//
//
//            int batchSize = 1000;
//            int iterations = 40;
//            int layerSize = 150;
//
//            Word2Vec vec = new Word2Vec.Builder()
//                    .batchSize(batchSize) //# words per minibatch.
//                    .minWordFrequency(2) //
//                    .useAdaGrad(false) //
//                    .layerSize(layerSize) // word feature vector size
//                    .iterations(iterations) // # iterations to train
//                    .learningRate(0.005) //
//                    .minLearningRate(1e-4) // learning rate decays wrt # words. floor learning
//                    .negativeSample(10) // sample size 10 words
//                    .iterate(iter) //
//                    .tokenizerFactory(tokenizer)
//                    .build();
//            vec.fit();
//            WordVectorSerializer.writeWordVectors(vec, path);
//        }
//        WordVectors model = WordVectorSerializer.loadTxtVectors(new File(path));
//
//        return model;
//    }
//
//    public static void main(String[] args) throws IOException {
//        WordVectors model = trian_model("data4/new-tweet/new-tweet.data");
//        System.out.println(model.similarity("like", "love"));
//    }
}
